/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package clusterAnalysis;

import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import jsc.independentsamples.MannWhitneyTest;
import jsc.tests.H1;
import org.uncommons.maths.random.MersenneTwisterRNG;
import units.CellType;
import units.MonkeyType;
import utils.FileUtils;
import utils.Handle;

/**
 *
 * @author jamie
 */
public class MinetClusterAnalysis {

    private File inputFolder = new File("/Users/jamie/Dropbox/monkeys/outputForR/minet");
    private File outputFolder = new File("/Users/jamie/Dropbox/monkeys/clust/minet/");
    private int iterations = 1000;
    private int minSize = 5;

    public MinetClusterAnalysis() {
        HashMap<String, MinetReader> minetData = getMinetData();
        TreeMap<Integer, HashMap<String, List<Integer>>> clusterData = getClusterData();
        TreeMap<String, HashMap<String, List<Double>>> results = getRealMinetResults(minetData, clusterData);
        MersenneTwisterRNG twister = new MersenneTwisterRNG();

        TreeMap<String, List<Double>> uValues_real = new TreeMap<String, List<Double>>();
        TreeMap<String, List<Double>> uValues_model = new TreeMap<String, List<Double>>();
        TreeMap<String, List<Double>> pValues_real = new TreeMap<String, List<Double>>();
        TreeMap<String, List<Double>> pValues_model = new TreeMap<String, List<Double>>();
        TreeMap<String, List<Double>> mean_real = new TreeMap<String, List<Double>>();
        TreeMap<String, List<Double>> mean_modelA = new TreeMap<String, List<Double>>();
        TreeMap<String, List<Double>> mean_modelB = new TreeMap<String, List<Double>>();
        for (int i = 0; i < iterations; i++) {
            System.out.println("iteration " + i);
            TreeMap<String, HashMap<String, List<Double>>> model = runModel(minetData, clusterData, twister, results);
            for (String comparison : results.keySet()) {
                if (!uValues_real.containsKey(comparison)) {
                    uValues_real.put(comparison, new ArrayList<Double>());
                    uValues_model.put(comparison, new ArrayList<Double>());
                    pValues_real.put(comparison, new ArrayList<Double>());
                    pValues_model.put(comparison, new ArrayList<Double>());
                    mean_real.put(comparison, new ArrayList<Double>());
                    mean_modelA.put(comparison, new ArrayList<Double>());
                    mean_modelB.put(comparison, new ArrayList<Double>());
                }
                List<Double> all_real = new ArrayList<Double>();
                List<Double> all_model = new ArrayList<Double>();
                List<Double> randomMinetValuesA = new ArrayList<Double>();
                List<Double> randomMinetValuesB = new ArrayList<Double>();
                for (String source : results.get(comparison).keySet()) {
                    all_real.addAll(results.get(comparison).get(source));
                    all_model.addAll(model.get(comparison).get(source));
                }
//                System.out.println(comparison+", "+all_real.size() + ", " + all_model.size());
                if (all_real.size() > minSize) {

                    double[] modelDoubles = asDoubles(all_model);
                    double[] realDoubles = asDoubles(all_real);
//                    System.out.println("waiting for stats...");
                    double meanReal = getMean(all_real);
                    double meanModel = getMean(all_model);
                    MannWhitneyTest mwut = new MannWhitneyTest(realDoubles, modelDoubles, H1.GREATER_THAN);
                    uValues_real.get(comparison).add(mwut.getStatistic());
                    pValues_real.get(comparison).add(mwut.getSP());
                    mean_real.get(comparison).add(meanReal);
                    mean_modelA.get(comparison).add(meanModel);
                    for (String source : results.get(comparison).keySet()) {

                        randomMinetValuesA.addAll(getRandomMinetValues(minetData.get(source), twister, results.get(comparison).get(source).size()));
                        randomMinetValuesB.addAll(getRandomMinetValues(minetData.get(source), twister, results.get(comparison).get(source).size()));
                    }
                    mean_modelB.get(comparison).add(getMean(randomMinetValuesB));
                    mwut = new MannWhitneyTest(asDoubles(randomMinetValuesA), asDoubles(randomMinetValuesB), H1.GREATER_THAN);
                    uValues_model.get(comparison).add(mwut.getTestStatistic());
                    pValues_model.get(comparison).add(mwut.getSP());
                }
            }
        }
        for (String comparison : uValues_real.keySet()) {
            
            if (uValues_real.get(comparison) != null) {
                if (uValues_real.get(comparison).size()>0) {
                    File out = new File(outputFolder, comparison + ".txt");
                    BufferedWriter writer = FileUtils.getBufferedWriter(out);
                    for (int i = 0; i < iterations; i++) {
                        try {
                            writer.append(uValues_real.get(comparison).get(i) + "\t" 
                                    +pValues_real.get(comparison).get(i)+"\t"
                                    +mean_real.get(comparison).get(i)+"\t"
                                    +mean_modelA.get(comparison).get(i)+"\t"
                                    +uValues_model.get(comparison).get(i) +"\t"
                                    +pValues_model.get(comparison).get(i) + "\t"
                                    +mean_modelB.get(comparison).get(i)+"\n");
                        } catch (IOException ex) {
                            Logger.getLogger(MinetClusterAnalysis.class.getName()).log(Level.SEVERE, null, ex);
                        }
                    }
                    try {
                        writer.close();
                    } catch (IOException ex) {
                        Logger.getLogger(MinetClusterAnalysis.class.getName()).log(Level.SEVERE, null, ex);
                    }
                }
            }
        }

    }

    private HashMap<String, MinetReader> getMinetData() {
        HashMap<String, MinetReader> minetData = new HashMap<String, MinetReader>();
        String[] sources = {"AL", "AP", "ML", "MP"};
        for (String source : sources) {
            File minetFile = new File(inputFolder, source + "_select_results.txt");
            MinetReader reader = new MinetReader();
            try {
                reader.readFile(minetFile);
            } catch (IOException ex) {
                Logger.getLogger(MinetClusterAnalysis.class.getName()).log(Level.SEVERE, null, ex);
            }
            minetData.put(source, reader);
        }
        return minetData;
    }

    private TreeMap<Integer, HashMap<String, List<Integer>>> getClusterData() {
        String[] sources = {"AL", "AP", "ML", "MP"};
        TreeMap<Integer, HashMap<String, List<Integer>>> clusterData = new TreeMap<Integer, HashMap<String, List<Integer>>>();
        Handle handle = Handle.getConnectedHandle("monkeys");
        String sql;
        for (String source : sources) {

            MonkeyType mt = MonkeyType.valueOf(source.substring(0, 1));
            CellType ct = CellType.valueOf(source.substring(1, 2));
            sql = "select clusterId, probeId from clusterMembership where monkeyType='" + mt.name() + "' and cellType='" + ct.name() + "' and clusterId!=0";
            ResultSet rs = handle.runSQLquery(sql);
            try {
                while (rs.next()) {
                    int clusterId = rs.getInt("clusterId");
                    int probeId = rs.getInt("probeId");
                    if (!clusterData.containsKey(clusterId)) {
                        clusterData.put(clusterId, new HashMap<String, List<Integer>>());

                    }
                    if (!clusterData.get(clusterId).containsKey(source)) {
                        clusterData.get(clusterId).put(source, new ArrayList<Integer>());
                    }
                    clusterData.get(clusterId).get(source).add(probeId);
                }
                rs.close();
            } catch (SQLException ex) {
                Logger.getLogger(MinetClusterAnalysis.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
        return clusterData;

    }

    private TreeMap<String, HashMap<String, List<Double>>> getRealMinetResults(HashMap<String, MinetReader> minetData, TreeMap<Integer, HashMap<String, List<Integer>>> clusterData) {
        Integer[] clusterIds = clusterData.keySet().toArray(new Integer[clusterData.keySet().size()]);
        TreeMap<String, HashMap<String, List<Double>>> results = new TreeMap<String, HashMap<String, List<Double>>>();
        for (int clusterIdA : clusterIds) {
            HashMap<String, List<Integer>> clusterDataA = clusterData.get(clusterIdA);
            for (int clusterIdB : clusterIds) {
                if (clusterIdA <= clusterIdB) {
                    System.out.println("getting real results for comparison: "+clusterIdA+"_"+clusterIdB);
                    HashMap<String, List<Integer>> clusterDataB = clusterData.get(clusterIdB);
                    results.put(clusterIdA + "_" + clusterIdB, new HashMap<String, List<Double>>());
                    for (String source : clusterDataA.keySet()) {
                        results.get(clusterIdA + "_" + clusterIdB).put(source, new ArrayList<Double>());
                        if (clusterDataB.containsKey(source)) {
                            List<Double> minetValues = getMinetValues(clusterDataA.get(source), clusterDataB.get(source), minetData.get(source));
                            results.get(clusterIdA + "_" + clusterIdB).get(source).addAll(minetValues);
                        }
                    }
                }
            }
        }
        return results;
    }

    private List<Double> getMinetValues(List<Integer> probeIdsA, List<Integer> probeIdsB, MinetReader minetReader) {
        List<Double> values = new ArrayList<Double>();
        for (int probeIdA : probeIdsA) {
            int indexA = Arrays.binarySearch(minetReader.getProbeIds(), probeIdA);

            for (int probeIdB : probeIdsB) {
                int indexB = Arrays.binarySearch(minetReader.getProbeIds(), probeIdB);
                if ((indexA < 0 || indexB < 0) || indexA == indexB) {
//                    System.out.println(indexA+", "+indexB+", "+probeIdA+", "+probeIdB);
                } else {
                    double[][] minetResults = minetReader.getMinetValues();
//                    System.out.println(indexA+":"+indexB);
                    double exp = minetResults[indexA][indexB];
                    values.add(exp);
//                    System.out.println(exp);
                }
            }
        }
        return values;
    }

    private TreeMap<String, HashMap<String, List<Double>>> runModel(HashMap<String, MinetReader> minetData, TreeMap<Integer, HashMap<String, List<Integer>>> clusterData, MersenneTwisterRNG twister, TreeMap<String, HashMap<String, List<Double>>> realResults) {
        Integer[] clusterIds = clusterData.keySet().toArray(new Integer[clusterData.keySet().size()]);
        TreeMap<String, HashMap<String, List<Double>>> results = new TreeMap<String, HashMap<String, List<Double>>>();
        for (int clusterIdA : clusterIds) {
            HashMap<String, List<Integer>> clusterDataA = clusterData.get(clusterIdA);
            for (int clusterIdB : clusterIds) {
                if (clusterIdA <= clusterIdB) {
//                    System.out.println("running: " + clusterIdA + "_" + clusterIdB);
                    HashMap<String, List<Integer>> clusterDataB = clusterData.get(clusterIdB);

                    results.put(clusterIdA + "_" + clusterIdB, new HashMap<String, List<Double>>());

                    for (String source : clusterDataA.keySet()) {
                        results.get(clusterIdA + "_" + clusterIdB).put(source, new ArrayList<Double>());
                        if (realResults.get(clusterIdA + "_" + clusterIdB).containsKey(source)) {
                            
                            if (clusterDataB.containsKey(source)) {
                                results.get(clusterIdA + "_" + clusterIdB).get(source).addAll(getModelMinetValues(clusterDataA.get(source), clusterDataB.get(source), minetData.get(source), twister, realResults.get(clusterIdA + "_" + clusterIdB).get(source).size()));
                                
                            }
                        }
                    }
                }
            }
        }
        return results;
    }

    private List<Double> getModelMinetValues(List<Integer> probeIdsA, List<Integer> probeIdsB, MinetReader minetReader, MersenneTwisterRNG twister, int size) {
        List<Double> values = new ArrayList<Double>();

        List<Integer> selectedProbeIds = getRandomProbeIds(probeIdsA, probeIdsB, size, twister, minetReader);

        for (int probeId : selectedProbeIds) {
            int indexA = Arrays.binarySearch(minetReader.getProbeIds(), probeId);
            int indexB = getRandomIndex(minetReader.getProbeIds(), indexA, twister);
            double[][] minetResults = minetReader.getMinetValues();
            double exp = minetResults[indexA][indexB];
            values.add(exp);
        }
//        System.out.println("returning " + values.size() + " values of " + size);
        return values;
    }

    private List<Integer> getRandomProbeIds(List<Integer> probeIdsA, List<Integer> probeIdsB, int size, MersenneTwisterRNG twister, MinetReader minetReader) {
//        System.out.println("getting " + size + " random probe Ids");
        List<Integer> all = new ArrayList<Integer>();
        all.addAll(probeIdsA);
        all.addAll(probeIdsB);
        List<Integer> selected = new ArrayList<Integer>();
        for (int i = 0; i < size; i++) {
            int random = all.get(twister.nextInt(all.size()));
            int index = Arrays.binarySearch(minetReader.getProbeIds(), random);
            if (index < 0) {
                i = i - 1;
            } else {
                selected.add(random);
            }
        }
//        System.out.println("returning " + selected.size() + " random probe Ids");
        return selected;
    }

    private int getRandomIndex(int[] probeIds, int notIndex, MersenneTwisterRNG twister) {
        int index = twister.nextInt(probeIds.length);
        if (index != notIndex) {
            return index;
        }
        return getRandomIndex(probeIds, notIndex, twister);
    }

    private double[] asDoubles(List<Double> list) {
        double[] doubles = new double[list.size()];
        int i = 0;
        for (double d : list) {
            doubles[i] = d;
            i++;
        }
        return doubles;
    }

    private double getMean(List<Double> doubles) {
        double t = 0;
        for (double d : doubles) {
            t += d;
        }
        return t / doubles.size();
    }

    private List<Double> getRandomMinetValues(MinetReader minetData, MersenneTwisterRNG twister, int size) {
        List<Double> values = new ArrayList<Double>();
        for (int i = 0; i < size; i++) {
            int indexI = twister.nextInt(minetData.getMinetValues().length);
            int indexJ = twister.nextInt(minetData.getMinetValues().length);
            if (indexI == indexJ) {
                i = i - 1;
            } else {
                values.add(minetData.getMinetValues()[indexI][indexJ]);
            }
        }
        return values;
    }
}
